package com.cike.sparkstudy.sql.java;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;

import java.util.List;

/**
 * 编程方式家在parquet格式数据
 */
public class ParquetLoadData {
    public static void main(String[] args){
        SparkConf conf = new SparkConf()
                .setMaster("local")
                .setAppName("ParquetLoadData");
        JavaSparkContext sc = new JavaSparkContext(conf);
        SQLContext sqlContext = new SQLContext(sc);

        //读取文件创建DataFrame
        DataFrame userDF = sqlContext.read().parquet("/developerCodes/test/users.parquet");
        //将DataFrame注册成表，执行sql
        userDF.registerTempTable("user");

        //执行sql
        DataFrame usernameDF = sqlContext.sql("select name from user");
        //对查询出来的数据进行transformation操作，处理数据，打印
        List<String> userNamesDF = usernameDF.javaRDD().map(new Function<Row, String>() {
            @Override
            public String call(Row row) throws Exception {
                return "Name:" + row.getString(0);
            }
        }).collect();

        for(String userName : userNamesDF){
            System.out.println(userName);
        }


    }
}
